from pymongo import MongoClient
from multiprocessing import Pool
from multiprocessing import Process
from selenium import webdriver
from bs4 import BeautifulSoup

import requests
import pprint
import random
import time
import json
import re

client = MongoClient('192.168.31.111', 27017, connect = False)
tang_poem_db = client['tang_poem_db']
tang_poem_coll = tang_poem_db['tang_poem_coll']


class Poem:
    def __init__(self, init_pages):
        for init_page in init_pages:
            self.init_page = init_page
            self.crawl_volume_index()

    def crawl_volume_index(self):
        r = requests.get(self.init_page)
        soup = BeautifulSoup(r.text)

        for item in soup.select('#DDlTang tr')[:-1]:
            for link in item.select('a'):
                url = link.get('href').split('_')[0]
                self.crawl_poem_list('http://www.chinapoesy.com/' + url)

    def crawl_poem_list(self, url):
        end = False

        for i in range(1, 1000):
            if end:
                break

            url_ = url + '_' + str(i) + '.html'
            print(url_)

            r = requests.get(url_)
            soup = BeautifulSoup(r.text)

            for item in soup.select('.TangAll'):
                title = item.select('li')[1].find(['b']).text.strip()
                lines = [re.sub('\r|\n|<b>|</b>|<p>|<br/>|</p>', '', str(i)) for i in item.select('li')[1].contents if re.findall('[\u4e00-\u9fa5]', str(i))]
                lines = [line.strip() for line in lines if line.strip() != title]
                first_line = lines[0]

                if not tang_poem_coll.find_one({'first_line': first_line}):
                    tang_poem_coll.insert_one({
                        'title': title,
                        'lines': lines,
                        'first_line': first_line
                    })

                    print(title)
                    print(lines)
                    print()
                else:
                    end = True


if __name__ == '__main__':
    Poem([
        'http://www.chinapoesy.com/TangshiAllindex.html',
        'http://www.chinapoesy.com/TangshiAllindex1.html',
        'http://www.chinapoesy.com/TangshiAllindex2.html'
    ])

    # print(tang_poem_coll.count())
